package com.wuwangfu.etl;

import com.alibaba.fastjson.JSON;
import com.wuwangfu.entity.LogInfo;
import com.wuwangfu.exactlyonce.KafkaStringSerializationSchema;
import com.wuwangfu.func.ToJsonMapFunc;
import com.wuwangfu.utils.FlinkUtils;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.core.fs.Path;
import org.apache.flink.formats.parquet.avro.ParquetAvroWriters;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

import java.time.ZoneId;
import java.util.Properties;

/**
 * @Description：实时ETL（拉取、过滤、关联维度、筛选字段、字段脱敏、数据拆分、转换格式或类型）
 * @Author：jcshen
 * @Date：2023-07-02
 *
 * 1、使用RichMapFunction关联维度数据（可以使用异步IO进行优化）
 * 2、将数据使用侧流（旁路）输出进行查询
 * 3、将数据写回到Kafka
 *    Kafka吞吐量高，且保证ExactlyOnce【FLinkKafkaProducer实现了分两阶段提交，继承了一个类TwoPhaseCommitSinkFunction  】
 *    实现了两个接口：CheckpointedFunction和CheckpointListener，可以保证checkpoint成功再提交事务
 * 4、将主流写入到HDFS，可以使用BulkSink，以Parquet格式写入
 *
 *   https://www.cnblogs.com/jj1106/p/13185378.html
 */
public class PreEtlAndTopic {
    public static void main(String[] args) throws Exception {

//        AnotherProperties anotherProperties = new AnotherProperties();
//        Properties kafkaProperties = anotherProperties.getKafkaProperties();

        System.setProperty("HADOOP_USER_NAME","hive");

        ParameterTool parameters = ParameterTool.fromPropertiesFile(args[0]);
        DataStream<String> lines = FlinkUtils.createKafkaStream(parameters, SimpleStringSchema.class);
        /**
         * 设置全局的参数
         *
         * flink中main方法中的参数设置成全局参数，就等价于将这些参数广播到TaskManager的各个slot中，以便subTask使用。
         * 不设置成全局参数，这些参数的引用也会随着subTask一起调度到slot中，供subTask使用，只是每次使用都需要网络IO进行获取，效率低。
         */
        FlinkUtils.env.getConfig().setGlobalJobParameters(parameters);
        //过滤数据
        SingleOutputStreamOperator<LogInfo> logInfo = lines.map(new ToJsonMapFunc());
        SingleOutputStreamOperator<LogInfo> filtered = logInfo.filter(new FilterFunction<LogInfo>() {
            @Override
            public boolean filter(LogInfo bean) throws Exception {
                return bean != null;
            }
        });
        /*将数据进行拆分，使用侧流输出：原来使用split方式，再select，现在使用侧流输出*/
        //流量的测输出流
        OutputTag<LogInfo> flowOutputTag = new OutputTag<LogInfo>("flow-output") {};
        //活动的Tag
        OutputTag<LogInfo> activityOutputTag = new OutputTag<LogInfo>("activity-output") {};

        SingleOutputStreamOperator<LogInfo> mainStream = filtered.process(new ProcessFunction<LogInfo, LogInfo>() {
            @Override
            public void processElement(LogInfo bean, ProcessFunction<LogInfo, LogInfo>.Context ctx, Collector<LogInfo> out) throws Exception {
                //根据数据所携带的具体类型进行判断
                String logType = bean.getLogType();
                if (logType.startsWith("act")) {
                    ctx.output(activityOutputTag, bean);
                } else {
                    ctx.output(flowOutputTag, bean);
                }
                //输出主流数据
                out.collect(bean);
            }
        });

        DataStream<LogInfo> activityStream = mainStream.getSideOutput(activityOutputTag);
        DataStream<LogInfo> flowStream = mainStream.getSideOutput(flowOutputTag);
        Properties properties = parameters.getProperties();

        //将侧流flowStream写入Kafka中
        String flowTopic = parameters.getRequired("flow.topic");

        //创建KafkaProducer，即flink中的Sink
        FlinkKafkaProducer<String> flowProducer = new FlinkKafkaProducer<>(
                flowTopic,
                new KafkaStringSerializationSchema(flowTopic),
                properties,//指定Kafka的相关参数
                FlinkKafkaProducer.Semantic.EXACTLY_ONCE//指定写入Kafka语义为EXACTLY_ONCE
        );

        flowStream.map(new MapFunction<LogInfo, String>() {
            @Override
            public String map(LogInfo logInfo) throws Exception {
                return JSON.toJSONString(logInfo);
            }
        }).addSink(flowProducer);

        //将侧流activityStream写入Kafka中
        String activityTopic = parameters.getRequired("activity.topic");
        //创建KafkaProducer，即flink中的Sink
        FlinkKafkaProducer<String> activityProducer = new FlinkKafkaProducer<>(
                activityTopic,
                new KafkaStringSerializationSchema(activityTopic),
                properties,
                FlinkKafkaProducer.Semantic.EXACTLY_ONCE
        );
        activityStream.map(new MapFunction<LogInfo, String>() {
            @Override
            public String map(LogInfo logInfo) throws Exception {
                return JSON.toJSONString(logInfo);
            }
        }).addSink(activityProducer);

        //将主流数据写入HDFS，并指定格式为parquet
        String path = parameters.getRequired("main.stream.hdfs.out.path");
        //指定文件目录生成的格式
        DateTimeBucketAssigner<LogInfo> bucketAssigner = new DateTimeBucketAssigner<>(
                "yyyy-MM-dd--HH-mm",
                ZoneId.of("Asia/Shanghai")
        );

        StreamingFileSink<LogInfo> streamingFileSink = StreamingFileSink.forBulkFormat(new Path(path),
                        ParquetAvroWriters.forReflectRecord(LogInfo.class))
                .withBucketAssigner(bucketAssigner)
                .build();

        //存储到HDFS中
        mainStream.addSink(streamingFileSink);

        FlinkUtils.env.execute();
    }
}
